{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入必要的工具包\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "import pickle\n",
    "\n",
    "#from .utils import FeatureEng\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "import scipy.spatial.distance as ssd\n",
    "from collections import defaultdict\n",
    "import locale\n",
    "import datetime\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#通用的编码函数\n",
    "class DataCleaner:\n",
    "\n",
    "  \"\"\"\n",
    "\n",
    "  Common utilities for converting strings to equivalent numbers\n",
    "\n",
    "  or number buckets.\n",
    "\n",
    "  \"\"\"\n",
    "\n",
    "  def __init__(self):\n",
    "\n",
    "    # 载入 locales\n",
    "\n",
    "    self.localeIdMap = defaultdict(int)##defaultdict给所有key赋予默认的value（int型为0）\n",
    "\n",
    "    for i, l in enumerate(locale.locale_alias.keys()):\n",
    "\n",
    "      self.localeIdMap[l] = i + 1\n",
    "\n",
    "    # 载入 countries\n",
    "\n",
    "    self.countryIdMap = defaultdict(int)\n",
    "\n",
    "    ctryIdx = defaultdict(int)\n",
    "\n",
    "    \n",
    "\n",
    "    # 载入 gender id 字典\n",
    "\n",
    "    self.genderIdMap = defaultdict(int, {\"male\":1, \"female\":2})\n",
    "\n",
    " \n",
    "\n",
    "  def getLocaleId(self, locstr):\n",
    "\n",
    "    return self.localeIdMap[locstr.lower()]\n",
    "\n",
    " \n",
    "\n",
    "  def getGenderId(self, genderStr):\n",
    "\n",
    "    return self.genderIdMap[genderStr]\n",
    "\n",
    " \n",
    "\n",
    "  def getJoinedYearMonth(self, dateString):\n",
    "\n",
    "    dttm = datetime.datetime.strptime(dateString, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
    "\n",
    "    return \"\".join([str(dttm.year), str(dttm.month)])\n",
    "\n",
    " \n",
    "\n",
    "  def getCountryId(self, location):\n",
    "\n",
    "    if (isinstance(location, str)\n",
    "\n",
    "        and len(location.strip()) > 0\n",
    "\n",
    "        and location.rfind(\"  \") > -1):\n",
    "\n",
    "      return self.countryIdMap[location[location.rindex(\"  \") + 2:].lower()]\n",
    "\n",
    "    else:\n",
    "\n",
    "      return 0\n",
    "\n",
    " \n",
    "\n",
    "  def getBirthYearInt(self, birthYear):\n",
    "\n",
    "    try:\n",
    "\n",
    "      return 0 if birthYear == \"None\" else int(birthYear)\n",
    "\n",
    "    except:\n",
    "\n",
    "      return 0\n",
    "\n",
    " \n",
    "\n",
    "  def getTimezoneInt(self, timezone):\n",
    "\n",
    "    try:\n",
    "\n",
    "      return int(timezone)\n",
    "\n",
    "    except:\n",
    "\n",
    "      return 0\n",
    "\n",
    " \n",
    "\n",
    "  def getFeatureHash(self, value):\n",
    "\n",
    "    if len(value.strip()) == 0:\n",
    "\n",
    "      return -1\n",
    "\n",
    "    else:\n",
    "\n",
    "      return int(hashlib.sha224(value.encode(\"utf8\")).hexdigest()[0:4], 16)\n",
    "\n",
    " \n",
    "\n",
    "  def getFloatValue(self, value):\n",
    "\n",
    "    if len(value.strip()) == 0:\n",
    "\n",
    "      return 0.0\n",
    "\n",
    "    else:\n",
    "\n",
    "      return float(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "#读取（1）中抽取的EVENT\n",
    "eventIndex = pickle.load(open(\"PE_eventIndex.pkl\",'rb'))\n",
    "n_events = len(eventIndex)\n",
    "\n",
    "print(\"number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wucwu1/anaconda3/lib/python3.7/site-packages/scipy/spatial/distance.py:698: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    }
   ],
   "source": [
    "#对（1）中抽取的EVENT利用通用的编码函数进行转化，结果进行聚类\n",
    "cleaner = DataCleaner()\n",
    "\n",
    "fin = open(\"events.csv\",'r')\n",
    "\n",
    "fin.readline()\n",
    "\n",
    "eventPropMatrix = ss.dok_matrix((n_events,7))\n",
    "\n",
    "eventContMatrix = ss.dok_matrix((n_events,101))\n",
    "\n",
    "for line in fin.readlines():\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    \n",
    "    if eventId in eventIndex:\n",
    "        i = eventIndex[eventId]\n",
    "        \n",
    "        eventPropMatrix[i,0] = cleaner.getJoinedYearMonth(cols[2])\n",
    "        eventPropMatrix[i,1] = cleaner.getFeatureHash(cols[3])\n",
    "        eventPropMatrix[i,2] = cleaner.getFeatureHash(cols[4])\n",
    "        eventPropMatrix[i,3] = cleaner.getFeatureHash(cols[5])\n",
    "        eventPropMatrix[i,4] = cleaner.getFeatureHash(cols[6])\n",
    "        eventPropMatrix[i,5] = cleaner.getFloatValue(cols[7])\n",
    "        eventPropMatrix[i,6] = cleaner.getFloatValue(cols[8])\n",
    "        \n",
    "        for j in range(9,109):\n",
    "            eventContMatrix[i,j-9] = cols[j]\n",
    "fin.close()\n",
    "\n",
    "eventPropMatrix = normalize(eventPropMatrix, norm=\"l2\",copy=False)\n",
    "sio.mmwrite(\"EV_eventPropMatrix\",eventPropMatrix)\n",
    "\n",
    "eventContMatrix = normalize(eventContMatrix,norm=\"l2\",copy=False)\n",
    "sio.mmwrite(\"EV_eventContMatrix\",eventContMatrix)\n",
    "\n",
    "eventPropSim = ss.dok_matrix((n_events,n_events))\n",
    "eventContSim = ss.dok_matrix((n_events,n_events))\n",
    "\n",
    "uniqueEventPairs = pickle.load(open(\"PE_uniqueEventPairs.pkl\",'rb'))\n",
    "\n",
    "for e1,e2 in uniqueEventPairs:\n",
    "    i = e1\n",
    "    j = e2\n",
    "    \n",
    "    if (i,j) not in eventPropSim:\n",
    "        epsim = 1 - ssd.correlation(eventPropMatrix.getrow(i).todense(),\n",
    "                                   eventPropMatrix.getrow(j).todense())\n",
    "        \n",
    "        eventPropSim[i,j] = epsim\n",
    "        eventPropSim[j,i] = epsim\n",
    "        \n",
    "    if (i,j) not in eventContSim:\n",
    "        ecsim = 1 - ssd.cosine(eventContMatrix.getrow(i).todense(),\n",
    "                              eventContMatrix.getrow(j).todense())\n",
    "        eventContSim[i,j] = ecsim\n",
    "        eventContSim[j,i] = ecsim\n",
    "        \n",
    "sio.mmwrite(\"EV_eventPropSim\", eventPropSim)\n",
    "sio.mmwrite(\"EV_eventContSim\", eventContSim)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
